wheel_builder.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308
  1. """Orchestrator for building wheels from InstallRequirements.
  2. """
  3. import logging
  4. import os.path
  5. import re
  6. import shutil
  7. from pip._internal.models.link import Link
  8. from pip._internal.operations.build.wheel import build_wheel_pep517
  9. from pip._internal.operations.build.wheel_legacy import build_wheel_legacy
  10. from pip._internal.utils.logging import indent_log
  11. from pip._internal.utils.misc import ensure_dir, hash_file, is_wheel_installed
  12. from pip._internal.utils.setuptools_build import make_setuptools_clean_args
  13. from pip._internal.utils.subprocess import call_subprocess
  14. from pip._internal.utils.temp_dir import TempDirectory
  15. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  16. from pip._internal.utils.urls import path_to_url
  17. from pip._internal.vcs import vcs
  18. if MYPY_CHECK_RUNNING:
  19. from typing import (
  20. Any, Callable, Iterable, List, Optional, Tuple,
  21. )
  22. from pip._internal.cache import WheelCache
  23. from pip._internal.req.req_install import InstallRequirement
  24. BinaryAllowedPredicate = Callable[[InstallRequirement], bool]
  25. BuildResult = Tuple[List[InstallRequirement], List[InstallRequirement]]
  26. logger = logging.getLogger(__name__)
  27. _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.IGNORECASE)
  28. def _contains_egg_info(s):
  29. # type: (str) -> bool
  30. """Determine whether the string looks like an egg_info.
  31. :param s: The string to parse. E.g. foo-2.1
  32. """
  33. return bool(_egg_info_re.search(s))
  34. def _should_build(
  35. req, # type: InstallRequirement
  36. need_wheel, # type: bool
  37. check_binary_allowed, # type: BinaryAllowedPredicate
  38. ):
  39. # type: (...) -> bool
  40. """Return whether an InstallRequirement should be built into a wheel."""
  41. if req.constraint:
  42. # never build requirements that are merely constraints
  43. return False
  44. if req.is_wheel:
  45. if need_wheel:
  46. logger.info(
  47. 'Skipping %s, due to already being wheel.', req.name,
  48. )
  49. return False
  50. if need_wheel:
  51. # i.e. pip wheel, not pip install
  52. return True
  53. # From this point, this concerns the pip install command only
  54. # (need_wheel=False).
  55. if req.editable or not req.source_dir:
  56. return False
  57. if not check_binary_allowed(req):
  58. logger.info(
  59. "Skipping wheel build for %s, due to binaries "
  60. "being disabled for it.", req.name,
  61. )
  62. return False
  63. if not req.use_pep517 and not is_wheel_installed():
  64. # we don't build legacy requirements if wheel is not installed
  65. logger.info(
  66. "Using legacy 'setup.py install' for %s, "
  67. "since package 'wheel' is not installed.", req.name,
  68. )
  69. return False
  70. return True
  71. def should_build_for_wheel_command(
  72. req, # type: InstallRequirement
  73. ):
  74. # type: (...) -> bool
  75. return _should_build(
  76. req, need_wheel=True, check_binary_allowed=_always_true
  77. )
  78. def should_build_for_install_command(
  79. req, # type: InstallRequirement
  80. check_binary_allowed, # type: BinaryAllowedPredicate
  81. ):
  82. # type: (...) -> bool
  83. return _should_build(
  84. req, need_wheel=False, check_binary_allowed=check_binary_allowed
  85. )
  86. def _should_cache(
  87. req, # type: InstallRequirement
  88. ):
  89. # type: (...) -> Optional[bool]
  90. """
  91. Return whether a built InstallRequirement can be stored in the persistent
  92. wheel cache, assuming the wheel cache is available, and _should_build()
  93. has determined a wheel needs to be built.
  94. """
  95. if req.editable or not req.source_dir:
  96. # never cache editable requirements
  97. return False
  98. if req.link and req.link.is_vcs:
  99. # VCS checkout. Do not cache
  100. # unless it points to an immutable commit hash.
  101. assert not req.editable
  102. assert req.source_dir
  103. vcs_backend = vcs.get_backend_for_scheme(req.link.scheme)
  104. assert vcs_backend
  105. if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir):
  106. return True
  107. return False
  108. assert req.link
  109. base, ext = req.link.splitext()
  110. if _contains_egg_info(base):
  111. return True
  112. # Otherwise, do not cache.
  113. return False
  114. def _get_cache_dir(
  115. req, # type: InstallRequirement
  116. wheel_cache, # type: WheelCache
  117. ):
  118. # type: (...) -> str
  119. """Return the persistent or temporary cache directory where the built
  120. wheel need to be stored.
  121. """
  122. cache_available = bool(wheel_cache.cache_dir)
  123. assert req.link
  124. if cache_available and _should_cache(req):
  125. cache_dir = wheel_cache.get_path_for_link(req.link)
  126. else:
  127. cache_dir = wheel_cache.get_ephem_path_for_link(req.link)
  128. return cache_dir
  129. def _always_true(_):
  130. # type: (Any) -> bool
  131. return True
  132. def _build_one(
  133. req, # type: InstallRequirement
  134. output_dir, # type: str
  135. build_options, # type: List[str]
  136. global_options, # type: List[str]
  137. ):
  138. # type: (...) -> Optional[str]
  139. """Build one wheel.
  140. :return: The filename of the built wheel, or None if the build failed.
  141. """
  142. try:
  143. ensure_dir(output_dir)
  144. except OSError as e:
  145. logger.warning(
  146. "Building wheel for %s failed: %s",
  147. req.name, e,
  148. )
  149. return None
  150. # Install build deps into temporary directory (PEP 518)
  151. with req.build_env:
  152. return _build_one_inside_env(
  153. req, output_dir, build_options, global_options
  154. )
  155. def _build_one_inside_env(
  156. req, # type: InstallRequirement
  157. output_dir, # type: str
  158. build_options, # type: List[str]
  159. global_options, # type: List[str]
  160. ):
  161. # type: (...) -> Optional[str]
  162. with TempDirectory(kind="wheel") as temp_dir:
  163. assert req.name
  164. if req.use_pep517:
  165. assert req.metadata_directory
  166. wheel_path = build_wheel_pep517(
  167. name=req.name,
  168. backend=req.pep517_backend,
  169. metadata_directory=req.metadata_directory,
  170. build_options=build_options,
  171. tempd=temp_dir.path,
  172. )
  173. else:
  174. wheel_path = build_wheel_legacy(
  175. name=req.name,
  176. setup_py_path=req.setup_py_path,
  177. source_dir=req.unpacked_source_directory,
  178. global_options=global_options,
  179. build_options=build_options,
  180. tempd=temp_dir.path,
  181. )
  182. if wheel_path is not None:
  183. wheel_name = os.path.basename(wheel_path)
  184. dest_path = os.path.join(output_dir, wheel_name)
  185. try:
  186. wheel_hash, length = hash_file(wheel_path)
  187. shutil.move(wheel_path, dest_path)
  188. logger.info('Created wheel for %s: '
  189. 'filename=%s size=%d sha256=%s',
  190. req.name, wheel_name, length,
  191. wheel_hash.hexdigest())
  192. logger.info('Stored in directory: %s', output_dir)
  193. return dest_path
  194. except Exception as e:
  195. logger.warning(
  196. "Building wheel for %s failed: %s",
  197. req.name, e,
  198. )
  199. # Ignore return, we can't do anything else useful.
  200. if not req.use_pep517:
  201. _clean_one_legacy(req, global_options)
  202. return None
  203. def _clean_one_legacy(req, global_options):
  204. # type: (InstallRequirement, List[str]) -> bool
  205. clean_args = make_setuptools_clean_args(
  206. req.setup_py_path,
  207. global_options=global_options,
  208. )
  209. logger.info('Running setup.py clean for %s', req.name)
  210. try:
  211. call_subprocess(clean_args, cwd=req.source_dir)
  212. return True
  213. except Exception:
  214. logger.error('Failed cleaning build dir for %s', req.name)
  215. return False
  216. def build(
  217. requirements, # type: Iterable[InstallRequirement]
  218. wheel_cache, # type: WheelCache
  219. build_options, # type: List[str]
  220. global_options, # type: List[str]
  221. ):
  222. # type: (...) -> BuildResult
  223. """Build wheels.
  224. :return: The list of InstallRequirement that succeeded to build and
  225. the list of InstallRequirement that failed to build.
  226. """
  227. if not requirements:
  228. return [], []
  229. # Build the wheels.
  230. logger.info(
  231. 'Building wheels for collected packages: %s',
  232. ', '.join(req.name for req in requirements), # type: ignore
  233. )
  234. with indent_log():
  235. build_successes, build_failures = [], []
  236. for req in requirements:
  237. cache_dir = _get_cache_dir(req, wheel_cache)
  238. wheel_file = _build_one(
  239. req, cache_dir, build_options, global_options
  240. )
  241. if wheel_file:
  242. # Update the link for this.
  243. req.link = Link(path_to_url(wheel_file))
  244. req.local_file_path = req.link.file_path
  245. assert req.link.is_wheel
  246. build_successes.append(req)
  247. else:
  248. build_failures.append(req)
  249. # notify success/failure
  250. if build_successes:
  251. logger.info(
  252. 'Successfully built %s',
  253. ' '.join([req.name for req in build_successes]), # type: ignore
  254. )
  255. if build_failures:
  256. logger.info(
  257. 'Failed to build %s',
  258. ' '.join([req.name for req in build_failures]), # type: ignore
  259. )
  260. # Return a list of requirements that failed to build
  261. return build_successes, build_failures